package com.gylang.crawler.crawler.dy2018;

import cn.hutool.core.util.StrUtil;
import com.gylang.crawler.crawler.EsLogDownloader;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Selectable;

import java.util.ArrayList;
import java.util.List;

/**
 * @author gylang
 * data 2021/5/27
 */
@Component
public class Dy2018CategoryProcessor extends BaseDy2018PageProcessor {

    @Autowired
    private Dy2018PageListProcessor pageListProcessor;
    @Autowired
    private EsLogDownloader esLogDownloader;
    @Override
    public void process(Page page) {


        // 提前数据 分类页面

        Selectable categoryList =page.getHtml().xpath("//*[@class='bd3l']/[@class='co_area2']/[@class='co_content2']/ul//a[@href]");
        List<Selectable> category = categoryList.nodes();
        // 新增分类基地址
        List<String> categoryUrlList = new ArrayList<>(category.size());
        for (Selectable selectable : category) {
            String href = selectable.$("a", "href").toString();
            if (StrUtil.isNotBlank(href) && !href.contains("html")) {
                String url = BASE_URL + href;
                categoryUrlList.add(url);

            }

        }
        Spider.create(pageListProcessor)
                .setDownloader(esLogDownloader)
                .addUrl(categoryUrlList.toArray(new String[0]))
                .thread(20)
                .run();
    }
}
